#!/bin/bash


#>> all ceph nodes' public network & cluster network complish
#>> ceph,ceph-radosgw install complish
    # dnf install -y ceph ceph-radosgw

#>> node1 ssh-key copied to ALL nodes
    #-[root@node1 ~]# ssh-keygen -t rsa
    #-[root@node1 ~]# ssh-copy-id root@node1
    #-[root@node1 ~]# ssh-copy-id root@node2
    #-[root@node1 ~]# ssh-copy-id root@node3



members=("node1" "node2" "node3")
monitor=("172.24.16.245" "172.24.16.246" "172.24.16.247")
pub_net="172.24.16.0/24"
cls_net="172.24.0.0/24"



#>> all ceph nodes' local domain resolv config
#=================================================================
#for((i=0; i<${#members[@]}; i++))
#do 
#for((j=0; j<${#members[@]}; j++))
#do
#	cmd="echo '${monitor[$j]}  ${members[$j]}' >> /etc/hosts"
#	ssh root@${members[$i]} $cmd
#done
#done
#=================================================================
ceph_fsid=`uuidgen`; echo "ceph cluster fsid -> $ceph_fsid"
mon_members=""; mon_hosts=""
for((i=0; i<${#members[@]}; i++))
do 
	mon_members=${mon_members}${members[$i]},
done
for((i=0; i<${#monitor[@]}; i++))
do 
	mon_hosts=${mon_hosts}${monitor[$i]}:6789,
done
mon_members=${mon_members%?}; mon_hosts=${mon_hosts%?} ##

cat > /etc/ceph/ceph.conf <<EOF
[global]
fsid = $ceph_fsid
mon_initial_members = $mon_members
mon_host = $mon_hosts
public_network = $pub_net
cluster_network = $cls_net
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
#osd_journal_size = 1024
#osd_pool_default_size = 2
#osd_pool_default_min_size = 1
#osd_pool_default_pg_num = 333
#osd_pool_default_pgp_num = 333
#osd_crush_chooseleaf_type = 1
mon_allow_pool_delete = true

EOF

#-------------------------------- generate keyring ----------------------------------

ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
ceph-authtool /tmp/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
chown ceph:ceph /tmp/ceph.mon.keyring
monmap_cmd="monmaptool --create"
for((i=0; i<${#members[@]}; i++))
do 
	monmap_cmd=${monmap_cmd}" --add "${members[$i]}" "${monitor[$i]}
done
monmap_cmd=${monmap_cmd}" --fsid "$ceph_fsid" /tmp/monmap"
echo `$monmap_cmd`

#----------------------------------- distribution keyring -----------------------------------------------
# >>> index = 1, from node2
for((i=1; i<${#members[@]}; i++))
do 
	scp /tmp/monmap root@${members[$i]}:/tmp/
	scp /tmp/ceph.mon.keyring root@${members[$i]}:/tmp/
	scp -r /etc/ceph/ root@${members[$i]}:/etc/ 
	ssh root@${members[$i]} chmod 644 /tmp/ceph.mon.keyring
done

#------------------------------------deploy monitor service, msgr2, MGR, MDS,  RGW -----------------------------------------------
# >>> index = 0, from node1
for((i=0; i<${#members[@]}; i++))
do 
	cmd="sudo -u ceph mkdir /var/lib/ceph/mon/ceph-${members[$i]}"
	ssh root@${members[$i]} $cmd
	cmd="sudo -u ceph ceph-mon --mkfs -i ${members[$i]} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring"
	ssh root@${members[$i]} $cmd
	cmd="systemctl start ceph-mon@${members[$i]}"
	ssh root@${members[$i]} $cmd
	cmd="systemctl enable ceph-mon@${members[$i]}"
	ssh root@${members[$i]} $cmd
done

ceph mon enable-msgr2  ###

for((i=0; i<${#members[@]}; i++))
do 
	cmd="sudo -u ceph mkdir /var/lib/ceph/mgr/ceph-${members[$i]}"
	ssh root@${members[$i]} $cmd
	cmd="ceph auth get-or-create mgr.${members[$i]} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${members[$i]}/keyring"
	ssh root@${members[$i]} $cmd
	cmd="systemctl restart ceph-mgr@${members[$i]}"
	ssh root@${members[$i]} $cmd
	cmd="systemctl enable ceph-mgr@${members[$i]}"
	ssh root@${members[$i]} $cmd
done

for((i=0; i<${#members[@]}; i++))
do 
	cmd="sudo -u ceph mkdir -p /var/lib/ceph/mds/ceph-${members[$i]}"
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="ceph auth get-or-create mds.${members[$i]} osd 'allow rwx' mds 'allow' mon 'allow profile mds'"
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="ceph auth get mds.${members[$i]} | tee /var/lib/ceph/mds/ceph-${members[$i]}/keyring"
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="echo ' ' >> /etc/ceph/ceph.conf" 
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="echo '[mds.${members[$i]}]' >> /etc/ceph/ceph.conf"
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="echo 'host = ${members[$i]}' >> /etc/ceph/ceph.conf" 
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="echo ' ' >> /etc/ceph/ceph.conf"
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="systemctl restart ceph-mds@${members[$i]}"
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="systemctl enable ceph-mds@${members[$i]}" 
	ssh root@${members[$i]} $cmd
	#echo $cmd
done

for((i=0; i<${#members[@]}; i++))
do
	cmd="mkdir -p /var/lib/ceph/radosgw/ceph-rgw.${members[$i]}"
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="chown -R ceph:ceph /var/lib/ceph/radosgw/ceph-rgw.${members[$i]}"
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="ceph auth get-or-create client.rgw.${members[$i]} osd 'allow rwx' mon 'allow rw'"
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="ceph auth get client.rgw.${members[$i]} | tee /var/lib/ceph/radosgw/ceph-rgw.${members[$i]}/keyring"
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="chown -R ceph:ceph /var/lib/ceph/radosgw/ceph-rgw.${members[$i]}"
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="systemctl start ceph-radosgw@rgw.${members[$i]}"
	ssh root@${members[$i]} $cmd
	#echo $cmd
	cmd="systemctl enable ceph-radosgw@rgw.${members[$i]}"
	ssh root@${members[$i]} $cmd
	#echo $cmd
done
## rgw daemons will active after OSD created.

ceph -s

#-------------------------------------- Copy bootstrap-osd keyring to other nodes -----------------------------------------
# >>> index = 1, from node2
for((i=1; i<${#members[@]}; i++))
do 
	scp /var/lib/ceph/bootstrap-osd/ceph.keyring root@${members[$i]}:/var/lib/ceph/bootstrap-osd/
done
